import datetime
import inspect
import json
import math
import random
import time
import types
import warnings

from re_common.baselibrary.tools.all_requests.aiohttp_request import AioHttpRequest
from re_common.baselibrary.tools.all_requests.httpx_requet import HttpxRequest
from re_common.baselibrary.tools.all_requests.mrequest import MRequest
from re_common.baselibrary.tools.all_requests.requests_request import RequestsRequest
from re_common.baselibrary.utils.basefile import BaseFile
from re_common.baselibrary.utils.basetime import BaseTime
from re_common.baselibrary.utils.core.mdeprecated import try_except2_async, retry_func_async
from re_common.baselibrary.utils.core.requests_core import USER_AGENTS, MsgCode

from apps.crawler_platform.core_abs.downstepabs import BaseCoreABS, deal_format, deal_eval, deal_load
from apps.crawler_platform.core_callback.core_callback import rrq_resp_hook
from apps.crawler_platform.core_callback.err_callback import all_client_back_callback
from apps.crawler_platform.core_platform.g_model import HtmlModel, MultiplePageHtmlModel, PageHtmlModel, TypeHtmlModel
from apps.crawler_platform.core_callback import *

STATICA_HACK = False
# globals()['kcah_acitats'[::-1].upper()] = False
if STATICA_HACK:  # pragma: no cover
    null()


class CoreDownMixin(BaseCoreABS):
    """
    下载功能
    由于整个下载的逻辑是以返回的第一个布尔值判断是否下载成功的，有时候下载失败
    需要继续下载，比如翻页和错误的状态码，所以抽取这个下载功能，继承类只实现简单的功能

    """

    def func_add(self, rrq=None):
        """
        rrq的方法添加器,通过callbakck的_m_list后缀添加
        :return:
        """
        if not rrq:
            rrq = self.rrq
        # 添加验证器
        name = "_".join([self.pm.task_name, self.pm.task_tag, "m", "list"])
        if name in globals():
            lists_m = eval(name)
            for item in lists_m:
                if item in self.pm.funcs:
                    func_m = types.MethodType(eval(item), rrq)
                    rrq.__dict__.update({item: func_m})

    async def _core_downler_aiohttp(self):
        self.rrq = AioHttpRequest()
        # 设置结果钩子函数
        self.rrq.set_resp_hook(rrq_resp_hook)
        # 附属信息，主要是钩子函数里面使用
        self.rrq.other_dicts = {"keys": self.pm.keys, "task_name": self.pm.task_name, "task_tag": self.pm.task_tag}
        # 添加验证器
        self.func_add()

        self.rrq.set_url(self.pm.url) \
            .set_timeout(self.pm.time_out) \
            .set_header(self.pm.this_header) \
            .set_resp_encoding(self.pm.resp_encoding) \
            .set_resp_errors(self.pm.resp_errors) \
            .set_allow_resp_text(self.pm.allow_resp_text) \
            .set_allow_resp_bytes(self.pm.allow_resp_bytes) \
            .set_force_close(self.pm.force_close)

        # self.rrq.set_skip_auto_headers(['User-Agent'])
        if self.pm.get_cookies and self.pm.cookies:
            self.rrq.set_cookies(self.pm.cookies)

        if self.pm.proxy:
            self.rrq.set_proxy(self.pm.proxy)
        # 验证器列表，由数据库配置
        Lists_middler = []
        if self.pm.resp_middler:
            self.rrq.set_middler_para(self.pm.resp_middler)
            for k, v in self.pm.resp_middler.items():
                Lists_middler.append(eval("self.rrq." + k))

        if self.pm.marks:
            self.rrq.set_marks(self.pm.marks)
            Lists_middler.append(self.rrq.marks_middlerwares)
        self.rrq.set_middler_list(Lists_middler)
        if self.pm.params:
            self.rrq.set_params(self.pm.get_params())
        if self.pm.req_type == "POST" and self.pm.post_data:
            data = self.pm.post_data
            if self.pm.post_data_is_json == "1":
                data = json.dumps(self.pm.post_data, ensure_ascii=False)
            self.rrq.set_data(data)
        if self.pm.is_grpc:
            self.rrq.set_data(self.pm.grpc_post_data)
        self.pm.time_use.print_time(self.pm.keys + "_down_before")
        if self.pm.req_type == "POST":
            bools, dicts = await self.rrq.run(MRequest.POST)
        elif self.pm.req_type == "GET":
            bools, dicts = await self.rrq.run(MRequest.GET)
        else:
            raise Exception("req_type 错误;" + self.pm.req_type)
        self.pm.time_use.print_time(self.pm.keys + "_down_after_" + str(self.pm.proxy))
        # 后置钩子函数
        return bools, dicts

    async def _core_downler_aiohttp_mu(self):
        # 用于并行请求多个url,cnki的特例化兼容
        rrq = AioHttpRequest()
        from app_main import request_ctx_var
        import asyncio
        dicts_p = request_ctx_var.get().state.ref_type_dicts[str(id(asyncio.current_task()))]
        dicts_p["rrq"] = rrq
        # 设置结果钩子函数
        rrq.set_resp_hook(rrq_resp_hook)
        # 附属信息，主要是钩子函数里面使用
        rrq.other_dicts = {"keys": self.pm.keys, "task_name": self.pm.task_name, "task_tag": self.pm.task_tag}
        # 添加验证器
        self.func_add(rrq)
        rrq.set_url(self.pm.url) \
            .set_timeout(self.pm.time_out) \
            .set_header(self.pm.this_header) \
            .set_resp_encoding(self.pm.resp_encoding) \
            .set_resp_errors(self.pm.resp_errors) \
            .set_allow_resp_text(self.pm.allow_resp_text) \
            .set_allow_resp_bytes(self.pm.allow_resp_bytes) \
            .set_force_close(self.pm.force_close)


        if self.pm.get_cookies and self.pm.cookies:
            rrq.set_cookies(self.pm.cookies)

        if self.pm.proxy:
            rrq.set_proxy(self.pm.proxy)
        # 验证器列表，由数据库配置
        Lists_middler = []
        if self.pm.resp_middler:
            rrq.set_middler_para(self.pm.resp_middler)
            for k, v in self.pm.resp_middler.items():
                Lists_middler.append(eval("rrq." + k))

        if self.pm.marks:
            rrq.set_marks(self.pm.marks)
            Lists_middler.append(rrq.marks_middlerwares)
        rrq.set_middler_list(Lists_middler)
        if self.pm.params:
            # print(self.pm.get_params())
            rrq.set_params(self.pm.get_params())
        if self.pm.req_type == "POST" and self.pm.post_data:
            data = self.pm.post_data
            # 如果需要替换
            if "post_data" in dicts_p.keys():
                data = dicts_p["post_data"]

            if self.pm.post_data_is_json == "1":
                data = json.dumps(data, ensure_ascii=False)
            rrq.set_data(data)

        if self.pm.req_type == "GET" and self.pm.params:
            data = self.pm.params
            # 如果需要替换
            if "params" in dicts_p.keys():
                data = dicts_p["params"]
            rrq.set_params(data)
        if self.pm.is_grpc:
            rrq.set_data(self.pm.grpc_post_data)
        self.pm.time_use.print_time(self.pm.keys + "_down_before")
        if self.pm.req_type == "POST":
            bools, dicts = await rrq.run(MRequest.POST)
        elif self.pm.req_type == "GET":
            bools, dicts = await rrq.run(MRequest.GET)
        else:
            raise Exception("req_type 错误;" + self.pm.req_type)
        # 后置钩子函数
        return bools, dicts

    async def _core_downler_httpx(self):
        self.rrq = HttpxRequest()
        # 设置结果钩子函数
        self.rrq.set_resp_hook(rrq_resp_hook)
        # 附属信息，主要是钩子函数里面使用
        self.rrq.other_dicts = {"keys": self.pm.keys, "task_name": self.pm.task_name, "task_tag": self.pm.task_tag}
        # 添加验证器
        self.func_add()

        self.pm.this_header["User-Agent"] = random.choice(USER_AGENTS)

        self.rrq.set_url(self.pm.url) \
            .set_timeout(self.pm.time_out) \
            .set_header(self.pm.this_header) \
            .set_resp_encoding(self.pm.resp_encoding) \
            .set_allow_resp_text(self.pm.allow_resp_text) \
            .set_allow_resp_bytes(self.pm.allow_resp_bytes)

        if self.pm.proxy:
            self.rrq.set_proxy(self.pm.proxy)
        # 验证器列表，由数据库配置
        Lists_middler = []
        if self.pm.resp_middler:
            self.rrq.set_middler_para(self.pm.resp_middler)
            for k, v in self.pm.resp_middler.items():
                Lists_middler.append(eval("self.rrq." + k))

        if self.pm.marks:
            self.rrq.set_marks(self.pm.marks)
            Lists_middler.append(self.rrq.marks_middlerwares)
        self.rrq.set_middler_list(Lists_middler)
        if self.pm.params:
            self.rrq.set_params(self.pm.get_params())
        if self.pm.req_type == "POST" and self.pm.post_data:
            data = self.pm.post_data
            if self.pm.post_data_is_json == "1":
                data = json.dumps(self.pm.post_data, ensure_ascii=False)
            self.rrq.set_data(data)
        if self.pm.is_grpc:
            self.rrq.set_data(self.pm.grpc_post_data)
        self.pm.time_use.print_time(self.pm.keys + "_down_before")
        if self.pm.req_type == "POST":
            bools, dicts = await self.rrq.run(MRequest.POST)
        elif self.pm.req_type == "GET":
            bools, dicts = await self.rrq.run(MRequest.GET)
        else:
            raise Exception("req_type 错误;" + self.pm.req_type)
        self.pm.time_use.print_time(self.pm.keys + "_down_after_" + str(self.pm.proxy))
        # 后置钩子函数
        return bools, dicts

    def _core_downler_requests(self):
        self.rrq = RequestsRequest()

        # 附属信息，主要是钩子函数里面使用
        self.rrq.other_dicts = {"keys": self.pm.keys, "task_name": self.pm.task_name, "task_tag": self.pm.task_tag}
        # 添加验证器
        self.func_add()

        self.rrq.set_url(self.pm.url) \
            .set_timeout(self.pm.time_out) \
            .set_header(self.pm.this_header)

        if self.pm.proxy:
            self.rrq.set_proxy(self.pm.proxy)
        # 验证器列表，由数据库配置
        Lists_middler = []
        if self.pm.resp_middler:
            self.rrq.set_middler_para(self.pm.resp_middler)
            for k, v in self.pm.resp_middler.items():
                Lists_middler.append(eval("self.rrq." + k))

        if self.pm.marks:
            self.rrq.set_marks(self.pm.marks)
            Lists_middler.append(self.rrq.marks_middlerwares)
        self.rrq.set_middler_list(Lists_middler)
        if self.pm.params:
            self.rrq.set_params(self.pm.get_params())
        if self.pm.req_type == "POST" and self.pm.post_data:
            data = self.pm.post_data
            if self.pm.post_data_is_json == "1":
                data = json.dumps(self.pm.post_data, ensure_ascii=False)
            self.rrq.set_data(data)
        if self.pm.is_grpc:
            self.rrq.set_data(self.pm.grpc_post_data)
        if self.pm.req_type == "POST":
            bools, dicts = self.rrq.run(MRequest.POST)
        elif self.pm.req_type == "GET":
            bools, dicts = self.rrq.run(MRequest.GET)
        else:
            raise Exception("req_type 错误;" + self.pm.req_type)
        # 后置钩子函数
        return bools, dicts

    async def _core_downler(self):
        """
        具体的下载逻辑,核心的下载器
        """
        if inspect.stack()[1].function != "_easy_down":
            raise Exception("不允许_easy_down以外的函数调用_core_downler")
        # 前置钩子函数
        await core_down_befor_callback(self)
        if self.pm.type_request == "aiohttp":
            # warnings.warn("在测试中使用 不要再主体程序中使用")
            # bools, dicts = await self._core_downler_httpx()
            if self.pm.task_tag in ("cnkiarticle","cnkithesisarticle","cnkicdfdthesisarticle","cnkiconferencearticle","cnkiipfdconferencearticle") and self.pm.keys in ("1_3", "1_4"):
                bools, dicts = await self._core_downler_aiohttp_mu()
            else:
                bools, dicts = await self._core_downler_aiohttp()
        elif self.pm.type_request == "httpx":
            bools, dicts = await self._core_downler_httpx()
        elif self.pm.type_request == "requests":
            warnings.warn("在测试中使用 不要再主体程序中使用，requests 会柱塞异步的fastapi")
            bools, dicts = self._core_downler_requests()
        else:
            warnings.warn("没有对应配置 默认 aiohttp")
            bools, dicts = await self._core_downler_aiohttp()
        self.pm.down_status.update({self.pm.keys: (bools, dicts)})
        core_down_after_callback(self.pm, self.rrq, bools, dicts)
        return bools, dicts

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    @retry_func_async(retry_times=3, sleep_time=1)
    async def _easy_down(self):
        """
        基础下载逻辑，不考虑其他逻辑，在里面获取代理，
        并且判断是否允许配置的code通过
        :param keys:
        :return:
        """
        self.step_info = self.add_all_process_info(f"获取代理,key is {self.pm.keys};")
        self.rrq = AioHttpRequest()
        # 配置为2表示同一个下载中代理用到失效再换
        state = self.pm.proxy_state
        if state == 2:
            if self.pm.proxy == "":
                bools, err_info = await self.get_proxy(self.pm.task_name, self.pm.task_tag)
                if not bools:
                    self.add_process_info(self.step_info + " 该步骤失败")
                    return bools, err_info
        else:
            bools, err_info = await self.get_proxy(self.pm.task_name, self.pm.task_tag)
            if not bools:
                self.add_process_info(self.step_info + " 该步骤失败")
                return bools, err_info
        self.step_info = self.add_all_process_info(f"下载网页,key is {self.pm.keys};" + str(self.pm.proxy))
        bools, err_info = await self._core_downler()
        # 下载遇到特殊的状态码允许通过，这些状态码在我们的配置当中
        if "code" in err_info.keys():
            # 设置下载的code状态，里面包含一定的判定,这里有重试3次的逻辑，且是最底层的状态码，所以使用覆盖，其他地方
            # 使用覆盖逻辑需要谨慎处理
            self.statemanager.set_keys_state(self.pm.keys, int(err_info["code"]), is_cover=True)
            if not bools:
                state_dicts = eval(self.pm.state_key)
                if str(err_info["code"]) in state_dicts["key_state"].split(","):
                    return True, err_info
                else:
                    self.pm.proxy = ""
                    return bools, err_info
        return bools, err_info

    async def order_down(self):
        """
        简单的排序下载，错误直接返回错误
        :return:
        """
        bools, err_info = await self._easy_down()
        if not bools:
            self.pm.set_down_err_info(err_info)
        self.pm.down_model.down_dict[self.pm.keys] = HtmlModel(
            html=self.rrq.html,
            state_dict=err_info,
            down_date=str(datetime.datetime.now())
        )
        # 对于不翻页 下载成功就设为True
        if bools:
            self.pm.set_is_his(True)
        return bools, err_info

    async def stat_down(self):
        """
        下载失败且code时会返回true，表示继续下载
        即不管code是否在我们的配置中，有code就通过
        使用场景: 配置了多个url，但有一些会下载失败，允许中间的url下载失败，极端情况下使用
        一般情况建议配置允许通过的code然后使用 order_down 调度
        :return:
        """
        self.step_info = self.add_all_process_info(f"通过stat_down下载;")
        bools, err_info = await self._easy_down()
        if not bools:
            self.pm.set_down_err_info(err_info)
        self.pm.down_model.down_dict[self.pm.keys] = HtmlModel(
            html=self.rrq.html,
            state_dict=err_info,
            down_date=str(datetime.datetime.now())
        )
        # 下载遇到有状态码允许通过，验证code的目的是防止程序异常，一般的url请求失败都有对应的code
        if "code" in err_info.keys():
            # 对于允许失败的情况下，任何下载结果都应该放到latest
            self.pm.set_is_his(True)
            return True, err_info
        return bools, err_info

    async def deal_down(self):
        """
        简单下载，下载完了直接解析并写数据库
        一般的下载有多个url时会循环下载完成后才会调用处理callback和save_sql,
        但是在该模式下每一个url都会调用一次callback
        :return:
        """
        self.step_info = self.add_all_process_info(f"deal_down下载")
        bools, err_info = await self._easy_down()
        if not bools:
            self.pm.set_down_err_info(err_info)
        self.pm.down_model.down_dict[self.pm.keys] = HtmlModel(
            html=self.rrq.html,
            state_dict=err_info,
            down_date=str(datetime.datetime.now())
        )
        if bools:
            self.pm.set_is_his(True)
        else:
            return bools, err_info
        self.step_info = self.add_all_process_info(f"deal_down中间过程调用deal_html()")
        bools, err_info = await self.deal_html()
        if not bools:
            self.add_process_info(self.step_info)
            return bools, err_info
        self.step_info = self.add_all_process_info(f"中间过程保存sql")
        bools, err_info = await self.save_sql()
        if not bools:
            return bools, err_info
        self.step_info = self.add_all_process_info(f"再次获取sql的单条数据")
        bools, err_info = await self.get_sql()
        if self.pm.sql_model.task_tag in ("cnkiarticle"):
            self.pm.sql_model.set_json(json.loads(self.deal_model.befor_dicts.update["article_info_json"]))
        if self.pm.sql_model.task_tag == "cnkiimagearticle":
            self.pm.sql_model.set_json(json.loads(self.deal_model.befor_dicts.update["article_json"]))
        if self.pm.sql_model.task_tag == "bookanjournalissue":
            self.pm.sql_model.set_json(json.loads(self.deal_model.befor_dicts.update["issue_json"]))
            # if "kcmslink" not in json.loads(self.pm.sql_model.article_info_json).keys():
            #     BaseFile.single_add_file("test_sql.txt", json.dumps(self.pm.sql_model.get_dict(),ensure_ascii=False) + "\n")

        if not bools:
            return bools, err_info
        return bools, err_info

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def list_multiple_page_down(self):
        """
        翻页，内部多个分类需要翻页，多个的来源不是同一个第一页，这是与5和6的主要区别，他们的请求是同一个url
        但是参数来之list，且每个参数内部有翻页需求
        :return:
        """
        import asyncio
        from app_main import request_ctx_var
        request_ctx_var.get().state.ref_type_dicts = {}
        if (self.pm.keys in ("1_3", "1_4") and ((self.pm.task_name == "cnkijournal" and self.pm.task_tag == "cnkiarticle") or self.pm.task_name in ("cnkithesis","cnkicdfdthesis","cnkiconference","cnkiipfdconference"))) :
            from apps.crawler_platform.core_callback.cnkijournal import down_ref_requests, deal_nxgpdata_request
            if BaseFile.is_file_exists(r"cnkiarticle_nxgpdata-request_1_3_ref_need.txt"):
                times = BaseFile.get_update_time(r"cnkiarticle_nxgpdata-request_1_3_ref_need.txt")
                if int(time.time()) - int(times) > 60 * 60 * 24:
                    await down_ref_requests(self)
            else:
                await down_ref_requests(self)
            list_type = deal_nxgpdata_request()
            self.pm.list_type = list_type

        # 初始化，防止down_dict没有keys
        self.pm.down_model.down_dict[self.pm.keys] = MultiplePageHtmlModel[HtmlModel, PageHtmlModel[HtmlModel]](
            first_page=HtmlModel(
                html="",
                down_date=str(datetime.datetime.now())
            ),
            type_page=TypeHtmlModel[PageHtmlModel[HtmlModel]](
                type_code={}
            )
        )
        size = 50
        async def fetch(type_p, page, v):
            dicts = self.pm.sql_model.get_dict()
            dicts["start"] = (page - 1) * size + 1
            dicts["type"] = type_p
            dicts["size"] = size
            dicts["v"] = v
            # bools, msg = deal_format(self.pm.one_para.task_info.post_data, dicts)
            bools, msg = deal_format(self.pm.one_para.task_info.params, dicts)
            if not bools:
                return bools, msg
            msg = deal_eval(msg)
            msg = deal_load(msg)
            request_ctx_var.get().state.ref_type_dicts[str(id(asyncio.current_task()))] = {}
            request_ctx_var.get().state.ref_type_dicts[str(id(asyncio.current_task()))]["params"] = msg
            await self._easy_down()
            dicts_p = request_ctx_var.get().state.ref_type_dicts[str(id(asyncio.current_task()))]
            rrq = dicts_p["rrq"]
            return (rrq.html, page, type_p, v)
        tasks = []
        dicts = self.pm.sql_model.get_dict()
        v = dicts["value"]
        for type_p in self.pm.list_type:
            task = asyncio.create_task(fetch(type_p, 1, v))
            tasks.append(task)
        # 等待所有请求完成
        responses = await asyncio.gather(*tasks)
        # 处理响应
        page_down = []
        err_down = []
        for response in responses:
            html, page, type_code ,v = response
            try:
                ref_dicts = json.loads(html)
                if ref_dicts["message"] == "success":
                    total = ref_dicts["data"]["total"]
                    for i in range(2, math.ceil(total / size) + 1):
                        page_down.append((type_code, i, v ))
            except:
                err_down.append((page, type_code))
                continue
            type_page = self.pm.down_model.down_dict[self.pm.keys].type_page
            if type_code not in type_page.type_code.keys():
                type_page.type_code[type_code] = PageHtmlModel(page_html={})
            type_page.type_code[type_code].page_html[str(page)] = HtmlModel(
                html=html,
                state_dict={},
                down_date=str(datetime.datetime.now())
            )

        tasks = []
        for type_p, i ,v in page_down:
            task = asyncio.create_task(fetch(type_p, i,v))
            tasks.append(task)
        # 等待所有请求完成
        responses = await asyncio.gather(*tasks)
        for response in responses:
            html, page, type_code,v = response
            try:
                ref_dicts = json.loads(html)
                if ref_dicts["message"] == "success":
                    total = ref_dicts["data"]["total"]
            except:
                err_down.append((page, type_code))
                continue
            type_page = self.pm.down_model.down_dict[self.pm.keys].type_page
            if type_code not in type_page.type_code.keys():
                type_page.type_code[type_code] = PageHtmlModel(page_html={})
            type_page.type_code[type_code].page_html[str(page)] = HtmlModel(
                html=html,
                state_dict={},
                down_date=str(datetime.datetime.now())
            )
        # print(self.pm.down_model.down_dict[self.pm.keys])
        # 设置保存nosql时判定是否下载完
        self.pm.set_is_his(True) # 如果不设置不会将数据放入latest
        if len(err_down) > 1:
            print(len(err_down))
            # 引文下载不全 可以通过 但需要对数据库打标
            self.statemanager.set_keys_state(self.pm.keys, MsgCode.PARE_NO_DATA_1, is_cover=True)
            return True, {"code": MsgCode.SUCCESS_CODE, "msg": "引文下载不全"}
        else:
            self.statemanager.set_keys_state(self.pm.keys, MsgCode.SUCCESS_CODE, is_cover=True)
            return True, {"code": MsgCode.SUCCESS_CODE, "msg": ""}

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def multiple_page_down(self):
        """
        翻页，内部有多个分类需要翻页，而不是只是简单的分页
        :return:
        """
        self.rrq = AioHttpRequest()
        self.rrq.html = None
        if self.pm.turn_page == 5:
            first_page = "1"
        elif self.pm.turn_page == 6:
            first_page = "0"
        else:
            raise Exception("翻页设计不兼容,请更改程序")
        # 初始化，防止down_dict没有keys
        self.pm.down_model.down_dict[self.pm.keys] = MultiplePageHtmlModel[HtmlModel, PageHtmlModel[HtmlModel]](
            first_page=HtmlModel(
                html=self.rrq.html,
                down_date=str(datetime.datetime.now())
            ),
            type_page=TypeHtmlModel[PageHtmlModel[HtmlModel]](
                type_code={}
            )
        )

        # 下载其中一页
        @try_except2_async(callback=all_client_back_callback, is_print=True)
        async def down_first_page(self, page):
            self.pm.set_post_data(str(page))
            self.pm.set_params(str(page))

            bools, err_info = await self._easy_down()
            if not bools:
                self.pm.set_down_err_info(err_info)
            if self.pm.keys not in self.pm.down_model.down_dict.keys():
                self.pm.down_model.down_dict[self.pm.keys] = MultiplePageHtmlModel[HtmlModel, PageHtmlModel[HtmlModel]](
                    first_page=HtmlModel(), type_page=TypeHtmlModel[PageHtmlModel[HtmlModel]](
                        type_code={}
                    ))
            self.pm.down_model.down_dict[self.pm.keys].first_page = HtmlModel(
                html=self.rrq.html,
                state_dict=err_info,
                down_date=str(datetime.datetime.now())
            )
            return bools, err_info

        # 下载其中一页
        @try_except2_async(callback=all_client_back_callback, is_print=True)
        async def down_one_page(self, type_code, page):
            type_page = self.pm.down_model.down_dict[self.pm.keys].type_page
            if str(type_code) in type_page.type_code.keys() and str(page) in type_page.type_code[
                str(type_code)].page_html.keys():
                html = type_page.type_code[str(type_code)].page_html[str(page)].html
                if html and len(html) > 0:
                    # 跳过
                    return True, "continue"

            self.pm.set_post_data(str(page))
            self.pm.set_params(str(page))
            bools, err_info = await self._easy_down()
            if not bools:
                self.pm.set_down_err_info(err_info)
            if bools:
                type_page = self.pm.down_model.down_dict[self.pm.keys].type_page
                if type_code not in type_page.type_code.keys():
                    type_page.type_code[type_code] = PageHtmlModel(page_html={})
                type_page.type_code[type_code].page_html[str(page)] = HtmlModel(
                    html=self.rrq.html,
                    state_dict=err_info,
                    down_date=str(datetime.datetime.now())
                )
            return bools, err_info

        # 保存和查询mongo的必要参数
        self.pm.set_para()
        bools, err_info = await self.read_nosql_html()
        if not bools:
            return bools, err_info
        # 查询结果
        data = self.pm.mongo_html["data"]

        self.pm.sql_model.page = first_page
        # 如果mongo查出来存在
        if data:
            # 注入到model
            data_obj = data[0]["new_data"]
            html = None
            # 防止new_data是空字典
            if data_obj:
                self.pm.down_model = self.pm.down_model.set_parse_obj(data_obj)
                # 第一页的html存在，且为当天的
                if self.pm.keys in self.pm.down_model.down_dict.keys():
                    first_html_model = self.pm.down_model.down_dict[self.pm.keys].first_page
                    if first_html_model.html != "" and BaseTime().get_beijin_date_strins(
                            format="%Y%m%d") < first_html_model.down_date:
                        html = self.pm.down_model.down_dict[self.pm.keys].first_page.html
                        # 启用第一页不信任机制，保证第一页会被重下 20210618
                        html = ""
                else:
                    html = ""
            # 如果第一页不存在
            if not (html and len(html) > 0):
                # 下载第一页,
                bools, err_info = await down_first_page(self, first_page)
                if not bools:
                    return bools, err_info
        else:
            # 不存在就 下载第一页
            bools, err_info = await down_first_page(self, first_page)
            if not bools:
                return bools, err_info
        # 解析第一页，得到页数
        bools, err_info = await self.deal_html()
        if not bools:
            return bools, err_info

        # 一个标志，判断页是否下载完
        is_his = True
        # 连续下载页数
        down_page_count = 0
        for k, v in self.deal_model.code_dicts[self.pm.keys].items():
            dicts = v["params"]
            page = v["max_page"]
            self.pm.sql_model.set_json(dicts)
            self.pm.sql_model.page = page
            # 启用连续3页下载错误便结束下载机制(20210618)目的是防止第一页网页可能给虚假的页数进行投毒
            page_down_fail = 0
            for page in range(int(first_page) + 1, int(self.pm.sql_model.page) + 1):
                bools, err_info = await down_one_page(self, k, page)
                if err_info != "continue":
                    down_page_count = down_page_count + 1
                if down_page_count >= 10:
                    bools, err_info = await self.save_nosql_html()
                    if not bools:
                        return bools, err_info
                    down_page_count = 0
                if not bools:
                    page_down_fail = page_down_fail + 1
                    is_his = False
                else:
                    page_down_fail = 0
                if page_down_fail > 3:
                    break
        # 设置保存nosql时判定是否下载完
        self.pm.set_is_his(is_his)
        return bools, err_info

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def page_down(self):
        #  TODO 不是好的解决办法,但能解决问题(其他人没有更好办法前不要改动以下两句代码)
        self.rrq = AioHttpRequest()
        self.rrq.html = None
        if self.pm.turn_page == 2:
            first_page = "1"
        elif self.pm.turn_page == 1:
            first_page = "0"
        else:
            raise Exception("翻页设计不兼容,请更改程序")
        # 初始化，防止down_dict没有keys
        self.pm.down_model.down_dict[self.pm.keys] = PageHtmlModel[HtmlModel](
            page_html={first_page: HtmlModel(
                html=self.rrq.html,
                down_date=str(datetime.datetime.now())
            )}
        )

        # 下载其中一页
        @try_except2_async(callback=all_client_back_callback, is_print=True)
        async def down_one_page(self, page):
            if self.pm.keys not in self.pm.down_model.down_dict:
                self.pm.down_model.down_dict[self.pm.keys] = PageHtmlModel[HtmlModel](
                    page_html={})
            page_html_dict = self.pm.down_model.down_dict[self.pm.keys].page_html
            # 如果page存在且不为空
            if str(page) in page_html_dict.keys() and page_html_dict[str(page)].html and len(
                    page_html_dict[str(page)].html) > 0:
                # 跳过
                return True, "continue"
            self.pm.set_post_data(str(page))
            self.pm.set_params(str(page))
            bools, err_info = await self._easy_down()
            if not bools:
                self.pm.set_down_err_info(err_info)
            # 对于翻页，一定是下载成功的才写入mongo,因为要依靠是否有html判断是否下载
            if bools:
                if self.pm.keys not in self.pm.down_model.down_dict:
                    self.pm.down_model.down_dict[self.pm.keys] = PageHtmlModel[HtmlModel](
                        page_html={})
                page_html = self.pm.down_model.down_dict[self.pm.keys].page_html
                page_html[str(page)] = HtmlModel(
                    html=self.rrq.html,
                    state_dict=err_info,
                    down_date=str(datetime.datetime.now())
                )
            return bools, err_info

        # 保存和查询mongo的必要参数
        self.pm.set_para()
        bools, err_info = await self.read_nosql_html()
        if not bools:
            return bools, err_info
        # 查询结果
        data = self.pm.mongo_html["data"]
        self.pm.sql_model.page = first_page
        # 如果mongo查出来存在
        if data:
            # 注入到model
            data_obj = data[0].get("new_data",{})
            html = None
            # 防止new_data是空字典
            if data_obj:
                self.pm.down_model = self.pm.down_model.set_parse_obj(data_obj)
                # 第一页的html
                if self.pm.keys in self.pm.down_model.down_dict and first_page in self.pm.down_model.down_dict[
                    self.pm.keys].page_html:
                    html = self.pm.down_model.down_dict[self.pm.keys].page_html[first_page].html
                    # 启用第一页不信任机制，保证第一页会被重下 20210618
                    html = ""
            # 如果第一页不存在
            if not (html and len(html) > 0):
                # 下载第一页,
                bools, err_info = await down_one_page(self, first_page)
                if not bools:
                    return bools, err_info
        else:
            # 不存在就 下载第一页
            bools, err_info = await down_one_page(self, first_page)
            if not bools:
                return bools, err_info
        # 解析第一页，得到页数
        bools, err_info = await self.deal_html()
        if not bools:
            return bools, err_info
        # 设置最大页
        self.pm.sql_model.page = self.deal_model.code_dicts[self.pm.keys]["max_page"]
        # 一个标志，判断页是否下载完
        is_his = True
        # 启用连续3页下载错误便结束下载机制 20210618
        page_down_fail = 0
        # 连续下载页数
        down_page_count = 0
        for page in range(int(first_page), int(self.pm.sql_model.page) + 1):
            bools, err_info = await down_one_page(self, page)
            if err_info != "continue":
                down_page_count = down_page_count + 1
            if down_page_count >= 10:
                bools, err_info = await self.save_nosql_html()
                if not bools:
                    return bools, err_info
                down_page_count = 0
            if not bools:
                page_down_fail = page_down_fail + 1
                is_his = False
            else:
                page_down_fail = 0
            if page_down_fail > 3:
                break

        # 设置保存nosql时判定是否下载完
        self.pm.set_is_his(is_his)
        return bools, err_info

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def part_page_down(self):
        self.rrq = AioHttpRequest()
        self.rrq.html = None
        # 保存和查询mongo的必要参数
        bools, err_info = await self.order_down()
        return bools, err_info
